{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# default_exp models.XCM"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# XCM (An Explainable Convolutional Neural Network for Multivariate Time Series Classification)\n",
    "\n",
    "> This is an unofficial PyTorch implementation by Ignacio Oguiza of  - oguiza@gmail.com based on Temporal Convolutional Network (Bai, 2018).\n",
    "\n",
    "**References:**\n",
    "\n",
    "* Fauvel, K., Lin, T., Masson, V., Fromont, É., & Termier, A. (2020). XCM: An Explainable Convolutional Neural Network for Multivariate Time Series Classification. arXiv preprint arXiv:2009.04796.\n",
    "* Official XCM PyTorch implementation: not available as of Nov 27th, 2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "from tsai.imports import *\n",
    "from tsai.utils import *\n",
    "from tsai.models.layers import *\n",
    "from tsai.models.utils import *\n",
    "from tsai.models.explainability import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "# This is an unofficial PyTorch implementation by Ignacio Oguiza - oguiza@gmail.com based on:\n",
    "\n",
    "# Fauvel, K., Lin, T., Masson, V., Fromont, É., & Termier, A. (2020). XCM: An Explainable Convolutional Neural Network for\n",
    "# Multivariate Time Series Classification. arXiv preprint arXiv:2009.04796.\n",
    "# Official XCM PyTorch implementation: not available as of Nov 27th, 2020\n",
    "\n",
    "class XCM(Module):\n",
    "    def __init__(self, c_in:int, c_out:int, seq_len:Optional[int]=None, nf:int=128, window_perc:float=1., flatten:bool=False, custom_head:callable=None, \n",
    "                 concat_pool:bool=False, fc_dropout:float=0., bn:bool=False, y_range:tuple=None, **kwargs):\n",
    "        \n",
    "        window_size = int(round(seq_len * window_perc, 0))\n",
    "        self.conv2dblock = nn.Sequential(*[Unsqueeze(1), Conv2d(1, nf, kernel_size=(1, window_size), padding='same'), BatchNorm(nf), nn.ReLU()])\n",
    "        self.conv2d1x1block = nn.Sequential(*[nn.Conv2d(nf, 1, kernel_size=1), nn.ReLU(), Squeeze(1)])\n",
    "        self.conv1dblock = nn.Sequential(*[Conv1d(c_in, nf, kernel_size=window_size, padding='same'), BatchNorm(nf, ndim=1), nn.ReLU()])\n",
    "        self.conv1d1x1block = nn.Sequential(*[nn.Conv1d(nf, 1, kernel_size=1), nn.ReLU()])\n",
    "        self.concat = Concat()\n",
    "        self.conv1d = nn.Sequential(*[Conv1d(c_in + 1, nf, kernel_size=window_size, padding='same'), BatchNorm(nf, ndim=1), nn.ReLU()])\n",
    "            \n",
    "        self.head_nf = nf\n",
    "        self.c_out = c_out\n",
    "        self.seq_len = seq_len\n",
    "        if custom_head: self.head = custom_head(self.head_nf, c_out, seq_len, **kwargs)\n",
    "        else: self.head = self.create_head(self.head_nf, c_out, seq_len, flatten=flatten, concat_pool=concat_pool, \n",
    "                                           fc_dropout=fc_dropout, bn=bn, y_range=y_range)\n",
    "\n",
    "            \n",
    "    def forward(self, x):\n",
    "        x1 = self.conv2dblock(x)\n",
    "        x1 = self.conv2d1x1block(x1)\n",
    "        x2 = self.conv1dblock(x)\n",
    "        x2 = self.conv1d1x1block(x2)\n",
    "        out = self.concat((x2, x1))\n",
    "        out = self.conv1d(out)\n",
    "        out = self.head(out)\n",
    "        return out\n",
    "    \n",
    "\n",
    "    def create_head(self, nf, c_out, seq_len=None, flatten=False, concat_pool=False, fc_dropout=0., bn=False, y_range=None):\n",
    "        if flatten: \n",
    "            nf *= seq_len\n",
    "            layers = [Flatten()]\n",
    "        else: \n",
    "            if concat_pool: nf *= 2\n",
    "            layers = [GACP1d(1) if concat_pool else GAP1d(1)]\n",
    "        layers += [LinBnDrop(nf, c_out, bn=bn, p=fc_dropout)]\n",
    "        if y_range: layers += [SigmoidRange(*y_range)]\n",
    "        return nn.Sequential(*layers)\n",
    "    \n",
    "    \n",
    "    def show_gradcam(self, x, y=None, detach=True, cpu=True, apply_relu=True, cmap='inferno', figsize=None, **kwargs):\n",
    "        \n",
    "        att_maps = get_attribution_map(self, [self.conv2dblock, self.conv1dblock], x, y=y, detach=detach, cpu=cpu, apply_relu=apply_relu)\n",
    "        att_maps[0] = (att_maps[0] - att_maps[0].min()) / (att_maps[0].max() - att_maps[0].min())\n",
    "        att_maps[1] = (att_maps[1] - att_maps[1].min()) / (att_maps[1].max() - att_maps[1].min())\n",
    "\n",
    "        figsize = ifnone(figsize, (10, 10))\n",
    "        fig = plt.figure(figsize=figsize, **kwargs)\n",
    "        ax = plt.axes()\n",
    "        plt.title('Observed variables')\n",
    "        im = ax.imshow(att_maps[0], cmap=cmap)\n",
    "        cax = fig.add_axes([ax.get_position().x1+0.01,ax.get_position().y0,0.02,ax.get_position().height])\n",
    "        plt.colorbar(im, cax=cax)\n",
    "        plt.show()\n",
    "\n",
    "        fig = plt.figure(figsize=figsize, **kwargs)\n",
    "        ax = plt.axes()\n",
    "        plt.title('Time')\n",
    "        im = ax.imshow(att_maps[1], cmap=cmap)\n",
    "        cax = fig.add_axes([ax.get_position().x1+0.01,ax.get_position().y0,0.02,ax.get_position().height])\n",
    "        plt.colorbar(im, cax=cax)\n",
    "        plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XCM(\n",
       "  (conv2dblock): Sequential(\n",
       "    (0): Unsqueeze(dim=1)\n",
       "    (1): Conv2dSame(\n",
       "      (conv2d_same): Conv2d(1, 128, kernel_size=(1, 51), stride=(1, 1))\n",
       "    )\n",
       "    (2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (3): ReLU()\n",
       "  )\n",
       "  (conv2d1x1block): Sequential(\n",
       "    (0): Conv2d(128, 1, kernel_size=(1, 1), stride=(1, 1))\n",
       "    (1): ReLU()\n",
       "    (2): Squeeze(dim=1)\n",
       "  )\n",
       "  (conv1dblock): Sequential(\n",
       "    (0): Conv1d(24, 128, kernel_size=(51,), stride=(1,), padding=(25,))\n",
       "    (1): BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (2): ReLU()\n",
       "  )\n",
       "  (conv1d1x1block): Sequential(\n",
       "    (0): Conv1d(128, 1, kernel_size=(1,), stride=(1,))\n",
       "    (1): ReLU()\n",
       "  )\n",
       "  (concat): Concat(dim=1)\n",
       "  (conv1d): Sequential(\n",
       "    (0): Conv1d(25, 128, kernel_size=(51,), stride=(1,), padding=(25,))\n",
       "    (1): BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (2): ReLU()\n",
       "  )\n",
       "  (head): Sequential(\n",
       "    (0): GAP1d(\n",
       "      (gap): AdaptiveAvgPool1d(output_size=1)\n",
       "      (flatten): Flatten(full=False)\n",
       "    )\n",
       "    (1): LinBnDrop(\n",
       "      (0): Linear(in_features=128, out_features=6, bias=True)\n",
       "    )\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tsai.data.all import *\n",
    "\n",
    "dsid = 'NATOPS'\n",
    "X, y, splits = get_UCR_data(dsid, split_data=False)\n",
    "tfms = [None, Categorize()]\n",
    "dls = get_ts_dls(X, y, splits=splits, tfms=tfms)\n",
    "model =  XCM(dls.vars, dls.c, dls.len)\n",
    "learn = Learner(dls, model, metrics=accuracy)\n",
    "xb, yb = dls.one_batch()\n",
    "\n",
    "bs, c_in, seq_len = xb.shape\n",
    "c_out = len(np.unique(yb.cpu().numpy()))\n",
    "\n",
    "model = XCM(c_in, c_out, seq_len, fc_dropout=.5)\n",
    "test_eq(model.to(xb.device)(xb).shape, (bs, c_out))\n",
    "model = XCM(c_in, c_out, seq_len, concat_pool=True)\n",
    "test_eq(model.to(xb.device)(xb).shape, (bs, c_out))\n",
    "model = XCM(c_in, c_out, seq_len)\n",
    "test_eq(model.to(xb.device)(xb).shape, (bs, c_out))\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/nacho/opt/anaconda3/envs/py36/lib/python3.6/site-packages/torch/nn/modules/module.py:974: UserWarning: Using a non-full backward hook when the forward contains multiple autograd Nodes is deprecated and will be removed in future versions. This hook will be missing some grad_input. Please use register_full_backward_hook to get the documented behavior.\n",
      "  warnings.warn(\"Using a non-full backward hook when the forward contains multiple autograd Nodes \"\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAnwAAAE1CAYAAAB9Uj1vAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAxYklEQVR4nO3de5hddX3v8c9379lzyUwyuV9IAgEJCCqCImKhFcULWh+x6lHwcrRHD+f0lB5t7QXteZTqqY/aVk/bh9oTK0XbCsV7qlSLiNL2yFUQCQhECCQhIZkkc7/uvb/nj71Sd8Jcvr+dTGb2yvvFsx9m9v7Mb/9mrbXX/PJbl6+5uwAAAJBfhbnuAAAAAGYXAz4AAICcY8AHAACQcwz4AAAAco4BHwAAQM4x4AMAAMg5BnzAcc7Mrjazv5/rfqQws3eb2b8do/f6kJn9TTB7nZn972ledzM79ej1DgBiGPABOZcNjn5qZsNmttvMPmtmi+e6X83C3T/u7u+d634AwJFgwAfkmJl9QNInJf2epG5J50s6SdLNZtZ6DPvRcqze62hq1n4DwOEY8AE5ZWaLJP2RpN9y9++4+4S7b5P0FkkbJL2jLt5uZv9oZgNm9mMze35dO39gZjuz1x42s4uz5wtmdpWZ/dzM9pnZjWa2NHttQ3b48j1m9qSk75vZP5vZlYf18Sdm9sbs62eb2c1mtj97n7fU5ZaZ2WYz6zezOyU9a5rfe6b3+XMz2561dY+Z/XJd7moz+4qZ/b2Z9Ut69+GHvM3sy9lMaZ+Z3WZmzzmsC8uz32PAzH5oZidN0c82M/tTM3vSzJ42s782s47steVm9i0z682Wx7+aGftrAA1jBwLk1y9Japf0tfon3X1Q0k2SXln39KWSvixpqaQvSfqGmZXM7HRJV0p6kbsvlPRqSduyn/ktSW+Q9FJJJ0g6IOmaw/rwUklnZD93vaTLD75gZmeqNtv4bTPrlHRz9t4rJV0m6a+yjLJ2RyWtkfRfssdUpnyf7Km7JJ1d97t+2czaD1sWX5G0WNI/TNL+P0vamPXzx5Nk3i7pY5KWS7pvijYk6ROSTsv6cqqktZI+nL32AUk7JK2QtErShyRRBxNAwxjwAfm1XFKPu5cneW1X9vpB97j7V9x9QtKnVRsoni+pIqlN0plmVnL3be7+8+xn/rukP3T3He4+JulqSW8+7DDo1e4+5O4jkr4u6ey6Ga+3S/pa9rOvk7TN3f/W3cvufq+kr0r6T2ZWlPQmSR/O2npA0hem+b2nex+5+9+7+77sff4s+/1Or/v5H7n7N9y9mvX7EO5+rbsP1P3Ozzez7rrIt939tuz1P5T0EjNbX9+GmZmkKyT9trvvd/cBSR9XbaArSROqDW5PymZm/9UpfA7gCDDgA/KrR7XDi5Odh7Yme/2g7Qe/cPeqarNLJ7j7VknvV21gs8fMbjCzE7LoSZK+nh127JX0kGoDxFVTtDug2izbwUHN5frF7NdJkl58sK2svbdLWq3aLFdLfVuSnpjql57hfWRmv2tmD2WHZHtVO7exfvBb/z6HMLOimX0iO4zdr1/Mdk7689ls6n7VZkDrrZC0QNI9db/vd7LnJelPJG2V9C9m9piZXTVVnwAgggEfkF8/kjQm6Y31T5pZl6TXSLql7un1da8XJK2T9JQkufuX3P1C1QZlrtpFIFJtYPMad19c92h395117R4+K3W9pMvN7CWqzSLeWtfWDw9rq8vdf0PSXknl+j5KOnGG333S98nO1/t91c5jXOLuiyX1SbJp+lzvbaod8n2FagPFDdnz9T9fvyy7VDt0/NRh7fRIGpH0nLrft9vdu6TaoNXdP+Dup0h6vaTfOXjuJAA0ggEfkFPu3qfaRRt/aWaXZOfkbZB0o2ozeH9XF3+hmb0xmw18v2oDxdvN7HQze7mZtal2Dt2IpGr2M38t6Y8PHjo1sxVmdukM3bpJtYHjRyX9YzabKEnfknSamb0z62fJzF5kZme4e0W18xCvNrMF2Tl572rwfRaqNnjcK6nFzD4sadEMbdVbqNqy2afaDN3HJ8m81swutNpV0B+TdLu7HzJrmPXnc5I+Y2YrJcnM1prZq7OvX2dmp2aHfvtUmzmtCgAaxIAPyDF3/5RqJ/z/qaR+SXeoNpt28cFz2jLflPRW1S68eKekN2bn87WpdnFBj6Tdql2o8MHsZ/5c0mbVDjsOSLpd0otn6M+YaoO3V6h2wcTB5wckvUq1w7BPZe/1yez9pdqFI13Z89dJ+ttG3kfSd1U7dPqIaoeFRzXNIdxJfDH7uZ2SHlTtdz7clyR9RLVDuS/UoVdD1/sD1Q7b3p4dHv6efnEu4cbs+0HVZmr/yt1vnbQVAAgwzgMGAADIN2b4AAAAco4BHwAAwDxiZtea2R4ze2CK183M/sLMtprZ/Wb2gpnaZMAHAAAwv1wn6ZJpXn+Nauf6blTtnp6fnalBBnwAAADziLvfptqFX1O5VNIXveZ2SYvNbM10bR7TwuAFK3mx0DZzUJIdclur6VUT7laQ0m5KtjJpMYPJuSrxrE8Ek/G+Fqw1/v6zVM2pmFCTPqUHrWqfOZSpJKyHYsK/jQpeDGc9Ydt1iy8J84T+Jm078WxLQrYavHgsvsakiseX7YSNh7PFhN1mRdHPr9Si+Oeyovj+JkUlvL+RChbfzosqhbOztT9P2c5TFBI+a9FPcNni68ES9k0lj2+7KZ/1lH10eyFlvxBvN+WeRcWETaGnvKfH3VdM9tqrLznL9/UMJrzzL9xzz+NbVLtLwEGb3H1TQhNrdegdBnZkz+2a6geO6YCvWGjT0o6zYtmEHcS4D8f7YPF2WxQbnErSQGV3ODteGQpnxyYOv1/r5CxhENfRun7mUKaa8AcgZfDSVVo1c+hguwl/uNfqtHB20PrD2c7a/XBDFiVkRzQ2cygzkfBHoM3j225XwnbeXojvMpa1xrMjldg6HijHh3y9z6yKNqXdxZ0zhzJd3j1zKNN3SDGT6a30deHsgcLecNYUH5j1VuLLoaMQXw6LtTKcHTvkb+D0SgmD5FaPZ1Ms8I5wthocGu0t7gm32ebxf+SuqC4LZ7sK8b+VEwn76I1d8fUwkvDvmuFKfHS4uDU+4vvc3r+YsqrPvp5B3XH3x8Jt1Wuxd4y6+7kN/XCDjumADwAAIA9crmo15bjDUbVTh1YfWpc9N6UjOocvu3v/w9lVItR6BAAAxwmXe7mhx1GwWdJ/zq7WPV9Sn7tPeThXOoIZPjMrSrpG0itVO3Z8l5ltdvcHG20TAACgKbhUq/x49JnZ9ZIukrTczHaoVr2nJEnu/teqlY98rWrVeoYl/fpMbR7JId3zJG1198eyzt2g2lUjDPgAAECuuVzVozNb98y23S+f4XWX9JspbR7JgG+yK0SeUUfTzK5Q7R4xSVeHAgAAzF9+tA7PHhOzftFGdpnxJkkqFbso3AsAAHLg+BnwJV8hAgAAkAvu8urxMeC7S9JGMztZtYHeZZLedlR6BQAAMN8dDzN87l42syslfVdSUdK17r7lqPUMAABg3jp+DunK3W9S7dLgkMWFTr1hwYtC2QUJPdsXL1agJQnXjSxri19uffe+54Sz2wt94ex9E9eHcqXi0nCb5xVeFs72KV7FpJpQ+uv5bfH+TgTLbknSWzfsC2cf6YtXHFm7IL4cTl+5feZQpncoXpVj/8iCcLa7Pd7fDau3hrOLlvWGs8teuS2cLT8eqwax9/6N4Ta37YhXrvi3p+LtntARrwTxYN+0pS0PccHK+LZ73/5Tw9lSIf75uXf/s8LZEzvDUZ23PL7P2zW8PJxd1hbf+Xe0xCvVFBOW2brulCpLsT9sP9yxIdxmyvZ4zrrHwtmVa54OZydG49V6Vv5OvCpH8eePhLOVHeGoCi86IZz93LR/Ll2qxreruUalDQAAgETux9EMHwAAwPHJpePkog0AAIDjkzPgAwAAyD8O6QIAAOSXyWXM8AEAAOQYh3QBAADyjgEfAABAzrmMc/gAAAByzCVV4wUa5hoDPgAAgGRctDGl/uqYvjccK+1SUCHc7piNhLOlsXgJmLaB9nB2n3aGs6PVeJmhWpnimU1UesMt3lm8NZwtV+PL1j1eMmeHrZ2Vdh/5WbzEXX9hVzi7qNodznY/dnI4O+zj4eyE4v+SLGlhOLvwgbPC2a6W2PYoScu/Ff8MjwdX8YFoUFJ/OV7y6KlCvDzWouqicLavEC+zd/v+leHsAYuXYSsm7Ob3WLw+1U/6Foeztx+Il0sb0VA426ZSOFtURzjripdWW1iIf94rwTKRT6s33GaHx/+mLX/y2eFsV/HMcDal/OWz/93C2b6JXw5nhxLGXUtb4/2VHp3mNWeGDwAAINe4ShcAACD/jBk+AACAHHMO6QIAAOQeM3wAAAC5xgwfAABArpk7M3wAAAC5x4APAAAgx5jhAwAAOA400YAvfit8AAAANKVjOsO3prWkD5246qi3u7h1LJxdvbA/nK16vDTT8Pj6cHb30BnxdssXhnL398bLwD06GC/n1V8YDWcHC4Ph7IgNh7NrqyeEs70WX7/tviCcLc7Sv42WFuPlnla3x8uadSZ8srta4mWGBsvxskj74h9LHRiP3a1+X0Kpvw2tXeHsGe3rwtnFpfjyWtMRL3tVsHi7HcXOcHbfWGs4u2vkeeHs06PxbWGoHC+Jt6wt/rlMkdAFLYovMlUSqnStao/NBnWX4uX7LGG72TUS348NTMTX70glnv2n/XvC2Z9X7gpnWyz+WVs0tiacnZ7Lqgkb1hzjkC4AAEAqV1Md0mXABwAAkIz78AEAAOSeOYd0AQAA8otaugAAAMcBLtoAAADIMXcGfAAAAHlHpQ0AAIBcY4YPAAAg31wM+AAAAPKNGb4pPTHao/c+dN2xfMtJxEvAtLeuDWfHJnrC2QUJ7Y5XYqXCLii9Ptzmr62LL4OnRxaGs30T8XJAr1kXL6/TMxL/QP3KGU+Es4vX7A1nO099KpwtrIuXufNd8RJzO35wTjjb0TUU74PHt4e+fUvC2TWnxtfFw/c9J5T76Z54SaTPb4svg+9N3B/OLijEl8H2Hd+Pt9u2IZwdHtsWzjadgbnuQJoNXa8OZ88c3BDK7UkoIdileEmx/3n6/nD27GdtDWcHB+Ol/k79dDgqb9kYzrbs3xXOFl/86/F2C7dM/SKVNgAAAPLNqKULAABwHGDABwAAkGNctAEAAJB3XLQBAACQby6p6nPdi7DCXHcAAACgKVWrjT1mYGaXmNnDZrbVzK6a5PUTzexWM7vXzO43s9fO1CYzfAAAAMlm55CumRUlXSPplZJ2SLrLzDa7+4N1sf8l6UZ3/6yZnSnpJkkbpmuXAR8AAECq2Tuke56kre7+mCSZ2Q2SLpVUP+BzSQdvftstacabxTLgAwAAaIQ3PMO33Mzurvt+k7tvyr5eK2l73Ws7JL34sJ+/WtK/mNlvSeqU9IqZ3pABHwAAQDI/khm+Hnc/9wje/HJJ17n7n5nZSyT9nZk9133qEegxHvCZLPiWraWVs9KDtmK8VFjBSrPSB1e8FEu5fCCUu7M4TfmXwzy+69Rwtqr4v17KPhbOfu9ny8PZMYuVl5Ok5Y8/O5wtW7xsT5vHt4VFFi+tNubxbWFY8eVrCSUEU7QpvhyW3XZWONtfngjlRhXLSdJTxZ3hbLkSX7YDlXhZwGKhO5zNs+h+X0rbPxYKXeFse2lZONti8XJl4x4vj/iIdoRyQ4XecJspff34wyeEs92PxEs5VhJmuU55aXz/WEmYPButxPfn3a13xhuezuwd0t0paX3d9+uy5+q9R9IlkuTuPzKzdknLJU25g+IqXQAAgEZUvbHH9O6StNHMTjazVkmXSdp8WOZJSRdLkpmdIald0rQF4o9ohs/MtqlW6roiqXyE05MAAABNwf1ITuGbrl0vm9mVkr4rqSjpWnffYmYflXS3u2+W9AFJnzOz31ZtrvHd7j7tSPJoHNJ9mbv3HIV2AAAAmscs3XjZ3W9S7VYr9c99uO7rByVdkNImF20AAACkcinhNPc5d6Tn8LlqlwXfY2ZXTBYwsyvM7O7a5cfNU4IEAABgWtUGH3PgSGf4LnT3nWa2UtLNZvYzd7+tPpDdV2aTJJkVGPEBAIB8aKJRzRHN8Ln7zuz/eyR9XbW7QwMAAOSbS161hh5zoeEBn5l1mtnCg19LepWkB45WxwAAAOa14+SQ7ipJXzezg+18yd2/c1R6BQAAMN/N0WxdIxoe8GVFfZ9/FPsCAADQHLJDus3imN6WpVTo1IqO2L2ZiwllzVJK25QSyl61ekc4uz+clMYq8VJhrnIoNzL+VLjNAwmleFLKpSVJ2PIqHi+n1aZ46aAhGwpnSx7v8Gx9/IsJZ2CklEDrLsazC4rxPqzqiC+JoXJsmzwwHu9r+8SGcHZnIf5Z76rGyzPuX7A2nF1TWR3O7ut8XjibYp/HSn9J0oLCknB2VSX+uRwoDISzHb4gnG3xYjibYmE1oVRY8Az/3YX456y9Gl8GSy2e7WqJL69KwoULy9vi4aFyfB9SLMSz3a1H60oLOz5m+AAAAI5rzoAPAAAgvzikCwAAcByoHmn9imOHAR8AAEAq5xw+AACA3PMmOoeveeYiAQAA0BBm+AAAABrBOXwAAAD55VylCwAAkHdctAEAAJB7zXTRxjEd8BXUooVaGst6/Lj4mOKlwto8XgYnJTtSSCmXVo23Gy7UFV9eZgklhhIq0LjHfy9L6W9CtpSS9dZwNqVUWUcxvny9Eo4mrYu2hHXcmlDGqaMlvnPraol3uGCxdkcr8b5OVOO7t5TyVF2Kl2Ebq3aFswsTSh5OVLvDWUso9jdWHAlnuxL6sCShpFdbNaF8nsXXcTG4jaVa2BLvQ9ljn4mR8uJwmx2K78eWtMb72lmKL6+J+K5fy9ripTJLhfh+rDgR3zcsKiV0eDouzuEDAADIO87hAwAAyDXjkC4AAECucUgXAAAg/zikCwAAkGMurtIFAADINzcO6QIAAOQdh3QBAAByjkO6AAAAecYhXQAAgPzjkO4UxquDemzs/4WyKWW6lFCqLKUEWYqqj4ez7vHSMtF6Wu6j4Rb7Rh5OeP+U2l9xo+NPhbOeUFNsb2FLvBMppeASSjjZeDzrXg5nU5gllK5L6G/B4mWvioPxkk/Rz3s1YXml7EOqPhbOFhJKoFUTPutbC7PTrilenqo6Hl8OKfvSlN+t2Vg5vhyq1dj2m7J+UxTLs7MeUj5rXx2Klyydrc97IWF/Pu17ikO6AAAA+ebM8AEAAOScyZ1z+AAAAPKNGT4AAIB84xw+AACAPOMcPgAAgHxzzuEDAADIP2b4AAAA8sw5hw8AACD3GPABAADkHId0p9BdWKKXL3hzKFu0+ELsbIlnF8YrQ6mYsB6HEypkDSVky9VYWbGBcrwEWl81Xgaup7A/nE0xoXgfllWXh7N9hd5wts3jJX6KCR+VksezwzYczq7wxfE+WLyc1pjHt51BjYSzVYuXxBuzWGnAMcVLCC6tLgtnyzY7JQQ7E7ax1oQSaEtK8R1ZJb4a1F+Ol/SqJJU8jO9HVlSXhrPVhD6MKf67dSheFvCA9YezC7wjlFtZXBBvsxi/aGC4klBuMJyUSgl/rx+t7Alnd2lrONtui8LZLi0OZx8Zf3zK17hoAwAAIO+4LQsAAED+NdM5fM0zFwkAADCPuFtDj5mY2SVm9rCZbTWzq6bIvMXMHjSzLWb2pZnaZIYPAAAgldusHNI1s6KkayS9UtIOSXeZ2WZ3f7Aus1HSByVd4O4HzGzlTO0y4AMAAEjkmrVDuudJ2uruj0mSmd0g6VJJD9Zl/quka9z9gCS5+4xXw3BIFwAAoAHuhYYekpab2d11jyvqml0raXvd9zuy5+qdJuk0M/t3M7vdzC6Zqa/M8AEAADSg2vgMX4+7n3sEb90iaaOkiyStk3SbmT3P3Xun+wEAAACkmKVz+CTtlLS+7vt12XP1dki6w90nJD1uZo+oNgC8a6pGOaQLAACQ6OA5fLNwle5dkjaa2clm1irpMkmbD8t8Q7XZPZnZctUO8T42XaPM8AEAADRgNi7acPeymV0p6buSipKudfctZvZRSXe7++bstVeZ2YOSKpJ+z933TdeuuSfU3TlCZuZKKB8091L6Gi9EUyh0hbO12dqZLe04I9zmKf7ccDalVFk1YRmcqnXh7GA1XhLpWR2x0kWStKQtHNXClvjv1lWKl+nqn4hvY7tH4hPypYS5+2D1PklpZQGXJizfp4Zjndg3MRZu8179KJwdntgbzqaYKMfbbSkuCWfLlQONdAezoFjoDmc7W1eHcqPlvnCbbS0Lw9lz7FfC2fXt8bKAown1+16+Ov4ZTjFRje/01iyIl7R86/1fvGeqc+2eu3ihf+XCF4bbqnfGt384ZbuzhRk+AACAVG6qNlEt3Rl7ambXmtkeM3ug7rmlZnazmT2a/T/+T1MAAAAcU5Gh6XWSDr+/y1WSbnH3jZJuyb4HAAA4Lrgkr1pDj7kw44DP3W+TtP+wpy+V9IXs6y9IesPR7RYAAMD8Nlu1dGdDo+fwrXL3XdnXuyWtmiqY3T36iqleBwAAaEZzNXhrxBFftOHuXrv6dsrXN0naJB28ShcAAKDJ+RFV2jjmGh3wPW1ma9x9l5mtkTRj0V4AAIC8cM3d4dlGNHo98WZJ78q+fpekbx6d7gAAADSHXJ3DZ2bXq1a+Y7mZ7ZD0EUmfkHSjmb1H0hOS3jKbnQQAAJhvcnVI190vn+Kli49yXwAAAJpGMx3SnceVNmanBJspYeVYyhHvUjzq8TJdUQdGHgln77GtR/39JckSzhDYlrRs4+6ciNfzcsVLoPksrLNau/FaZZ5Qui5aki9dvA+W8JmIrgtL2C+4ZmkZzNK2QLm0+ST+d6Lq8VJhQ+O7Q7mUz3o1odzgj2xzOHtHOeFvWoLvPBEvf5myj64m7EuPFncGfAAAADln+TqkCwAAgGdihg8AACDnGPABAADkmCtnV+kCAADgMFy0AQAAkHdctAEAAJBrrlp5tWbBgA8AAKABHNIFAADIOQ7pTqFU6NLyjheEsi0Wr5gw7sPxPlh7ONuieB8OVLaHs2OV/nB2dHxHLGit4TYXtp4czlaSKkHE74q+uGV9vA8JFRNOrZ4RzvZafD0s8AXh7BKL30l+1OPLbELxdVFK+Gh3F+N31O9siVdIWdEe3xEOB3+1A+PxCgQHJuLbzc7C0+FsV3VhOLu/uC+cXVNZHc7uK+wPZ1Ps8+D+RlJHoTucXV5dE84O22A42+bx/Xmbx/fnKboS/k5Et97dhT3hNlOWwWotCWcXtsSr2kxUPZw9ozu+DxmpHP19iCQta4v391M7/3KaV40ZPgAAgDxzZ4YPAAAg95jhAwAAyLkqV+kCAADkl4sZPgAAgJzjxssAAAC5xwwfAABAjrnit9qZDxjwAQAApHJm+AAAAHKPc/gAAAByzrkty+QWqEPn2vNC2fZivPxK70S8psqiYkLJqdb4inxoKF4WaVfr7nB2W7C0WrHQGW5zfeG54exgoS+crSaczXBa9Vnh7ITH2331qvj63T4cX2cr2uJ9OLFzNJztn4iXNeubSCif1xIv2bauM17KallHvIzhaRseD2f7emNlurbtXRVuc8dQVzj74/0nhrMr2uNlmbYNLg5nz1oRX2dbB+L9bUn4e/SzwZXh7JpivKTYmcvin5+esaXh7OJSvN2WQny9tVg8u6pjLJwdr8TKld2TsD2mlAl7zuL4Z33VgqFwNvp7SdIF598ZzvbvWRbODgzEP++r1z8Vzn7qb6Z+zblKFwAAIP8SygjPOQZ8AAAADeCQLgAAQI65uGgDAAAg31xyDukCAADkW5VDugAAAPnl4sbLAAAAOcdtWQAAAHKviU7hY8AHAACQqtmu0o2XswAAAEBTOqYzfEMa0o/8rlC2UI6XaikrXtqmpRIvB9QyEs8O+p5wdnw8XrImqlIZCGcfHf/3cLbq8WWbYl/LY7PS7pM9p4ezIx4vG9c+HC/b07UvViZMksYsXoZtzEbC2ZLHt90uXxTOdnh8OSzbcl44OxYsn9fv8eU1av3hbI/9LJxtG40vg5Rt7Cd71oSzfeoJZ4sWL9/Xr3jZx5+V48vh3p54SbwRi+8f27wjnE1RSJgLafcF4WxFsTKgByz+96Q0Gv+sLzmwIpxN+ayXE8pqnrj1teHsUCVebnC0Gi+x2t1yZjgrTb9viP/mc49DugAAAA1opqt0OaQLAACQyL12Dl8jj5mY2SVm9rCZbTWzq6bJvcnM3MzOnalNBnwAAAAN8AYf0zGzoqRrJL1G0pmSLjezZxyHNrOFkt4n6Y5IXxnwAQAANGCWZvjOk7TV3R9z93FJN0i6dJLcxyR9UlLo5GYGfAAAAIlctYs2GnlIWm5md9c9rqhreq2k7XXf78ie+w9m9gJJ693929H+ctEGAABAMjuSizZ63H3G8+4mfVezgqRPS3p3ys8x4AMAAGjALN2WZaek9XXfr8ueO2ihpOdK+oGZSdJqSZvN7PXufvdUjTLgAwAASOSatduy3CVpo5mdrNpA7zJJb/uP93Xvk7T84Pdm9gNJvzvdYE9iwAcAANCQ6iwU03X3spldKem7koqSrnX3LWb2UUl3u/vmRtplwAcAANCAWRjv1dp1v0nSTYc99+EpshdF2jymA74VxU69d2ms3FKxEF+My1rjJVUWt46Hs0WLH50fKp8azu4daw1nx6ux6eLHB+Kl6HaNToSzexQv2Tah+LJNKSm2qroynN3vveHs0uqycHaB2sPZrkK8lFXV49v50lL841pIOMpQsni4dyL+mRisxD+X0ZJpYxbfxtba0nD2RItvC20JC3d1R/xGCB3F+LawqBQvVTZUjvfh6ZEzwtm+hG2htxxfb8tK8VJhlYTPz3g13t/Olvj+dLQSb3dFe6zd5W3x9duW8Ldy92h8W6gkjGTG4hXQdN9YvHzf9uoD4WypEC9xt9Djf1Omc/DGy82CGT4AAIAGUEsXAAAg53JVS9fMrjWzPWb2QN1zV5vZTjO7L3u8dna7CQAAMH8c4Y2Xj7nIAf3rJF0yyfOfcfezs8dNk7wOAACQW+6NPebCjId03f02M9twDPoCAADQNKrK0SHdaVxpZvdnh3yXTBUysysO1oobro4cwdsBAADMD67affgaecyFRgd8n5X0LElnS9ol6c+mCrr7Jnc/193PXVDoaPDtAAAA5pdcHdKdjLs/ffBrM/ucpG8dtR4BAADMe5b/Q7pmtqbu21+TFL87IgAAQLNrcHZv3s7wmdn1ki6StNzMdkj6iKSLzOxs1Q5hb5P032aviwAAAPPLwduyNIvIVbqXT/L05xt5s10Te/Wx7Z9t5EfnREtxymtRnqFSHQpnO1pPCGfLlVjJqbNKrwi3+Uvd3eHskpH4MhhOKDF04cp4tn8iXubol1bGs6u794SzS5b0hrNdS/rC2f6e+PLdunN9ONtRipeyKlfiy6xnpDOcPXHx/nD2gT2rQ7ltQ13hNr+y/6lwdnfl0XBWCWWk+vofDGfbSvH9wthE/HdrOk12bV9n27PC2VOqLwjl9o3Ey48tSSgT9pYVK8LZc5bFP79DE/FyoZ970/fCWVl8KqwyHC9/2X7Ow+Fs6bLpX5+rCzAaQaUNAACABjTReI8BHwAAQKrabVma56INBnwAAAANmKsLMBrBgA8AAKABubpoAwAAAIdyMcMHAACQe8zwAQAA5Nkc1sVtBAM+AACARC5uywIAAJB7zPABAADkHBdtTCt2k8JiYVG4RU84bbJg8RIwLcWOcNYsviij5dIkaaLSG8o9UPhBuM3tA+vC2bHqYDhb9Ylw9s6nE8rWKd7u9fvjZYYKakvIrgln2z1eamnM4tvCqMVLts2WqvrD2cXVZeHsQCFWKqyssXCbfZWd4exouTecLVgpnJXiZesmKgMJ7aImftPblL8pKfvzghXC2ae0NZQbr8T3u8N2IJz9Qs9wOHvj3gXhbNXipUWv/dRb4+0mHDCd8Pg4oKuQMvT5qylfyV0tXQAAADwTh3QBAAByronGewz4AAAAUtVq6c51L+IY8AEAAKRyLtoAAADIvWa6aCN+eREAAACaEjN8AAAAiTiHDwAA4DjQROM9BnwAAACNYIYPAAAg57hKdwqlQpdWdJwby1q87NWox8vQtFtXONvm8dJqe6qPh7NjlXh5qvFyrPTW2MSecJujxXiJoZRyaSlaLV62p5LQhxOrJ4SzvRYvM9SVsC0sKcbL941W49d4jVUr4WybxUt6LS7FdwOdLfFSVivb43vCoXKsJN6B8XibPdXTwtntbXvD2a5qfB+yr7UnnD2hGi/ft6cQ768lXJvX40+Gs12FeOm8lZX479ZfiO8fOzy+H2nzlJJ4cYusPZwtB8t/PVWMr982j7//Oi0OZxe1JpQFTLhU9TmL45/hkUp8fzNcjmdXtMf3pTdNUzWO0moAAADHgWoTTfEx4AMAAGhA8wz3GPABAAAkc+eiDQAAgJxzeRPN8THgAwAASMSNlwEAAI4DXKULAACQc85VugAAAPnFffgAAACOA800wxe/BTsAAAD+Q7XBx0zM7BIze9jMtprZVZO8/jtm9qCZ3W9mt5jZSTO1eUxn+BbaAl3cdlYo25owFO1NKLe0sBQvv7IooRLPlv7V4ezulv3xdsd3hnItxe5wm6fpheFsfzFe5qiicjh7htYntBtfv7+6Pt6HxwaXhLMndMTb3dgdX7/7R+Ml2wYm4htkZ8t4OLt+UbyM0/KF8e3hlLMeCmcHn46V6dq+fW24ze29S8PZ23vipb9StoWfD54Szr5g6TQ1nA7zSH/885NQDU9b+uKlCVe1x3fSZy+JlYiUpH3j8XWxqBRfF+3FeDmtlJmQVQt6w9mKx1q+a298O1/eHi89+bxl8RKcK7sPhLPj4/Fyks9/0y3h7OiTy8PZoT3xz/uSZz8Rzl79walfq12le/Rn+MysKOkaSa+UtEPSXWa22d0frIvdK+lcdx82s9+Q9ClJb52uXWb4AAAAGuAN/jeD8yRtdffH3H1c0g2SLj3kfd1vdfeDBeFvl7RupkY5hw8AAKABR3DRxnIzu7vu+03uvin7eq2k7XWv7ZD04mnaeo+kf57pDRnwAQAAJHK5qo1X2uhx93OPtA9m9g5J50p66UxZBnwAAACpfHbO4ZO0UzrkRPd12XOHMLNXSPpDSS9197GZGmXABwAA0IBZqqV7l6SNZnayagO9yyS9rT5gZudI+r+SLnH30NU4DPgAAAAS1W68fPQHfO5eNrMrJX1XUlHSte6+xcw+Kulud98s6U8kdUn6splJ0pPu/vrp2mXABwAA0IDZGPBJkrvfJOmmw577cN3Xr0htkwEfAABAstAtVuYNBnwAAACJZuuQ7mxhwAcAAJDKpKodwZ34jjE7loV/S8UuX9oRK61mCUVAJnwknG2xtnhW8exgJV6yZqIa7+9IsLRarRJLTKkYL0FTnflK74a0tsRLwaVYVoqXshrxvnC2w+L97fJ4dsSGZw5lKoqXUCoqoQxbtSucbfP4Z2KJdYazFY/tNPuV8NkpxLP7tSucTdkWUraxpYqXFBu0eLuFhH3pQDVeZq/N4tvNQsX3OeMWLwvY6vGSXrNlgceXw4Riv1t/IV7WrJTwmVxcjZeT7Ej4+zeeUFZzfSm+vIYr8cHUcDXeh+5ifP/4T4PX3DPV/fI6i8v92e2/Gm6r3o+Hvzhlu7OFGT4AAIBEnt16uVnM+E8/M1tvZrea2YNmtsXM3pc9v9TMbjazR7P/x//pAAAA0OSqWbWN1MdciMz1lyV9wN3PlHS+pN80szMlXSXpFnffKOmW7HsAAIDjQtWqDT3mwoyHdN19l1Q7wcXdB8zsIdUK+14q6aIs9gVJP5D0B7PSSwAAgHmkdkC3eQ7pJp3DZ2YbJJ0j6Q5Jq7LBoCTtlrTq6HYNAABg/srlgM/MuiR9VdL73b0/K+UhSXJ3N7NJD0qb2RWSrpCkgs39FVUAAABHrrku2ggN+MyspNpg7x/c/WvZ00+b2Rp332VmayRNel8Sd98kaZNUuy3LUegzAADAnHI11334IlfpmqTPS3rI3T9d99JmSe/Kvn6XpG8e/e4BAADMR7Vz+Br5by5EZvgukPROST81s/uy5z4k6ROSbjSz90h6QtJbZqWHAAAA85CrMtddCItcpftvkmyKly8+ut0BAACY/3J9le6ROqHUoavWPi+UnfCpxpjP1NmSUFKlFC/b01GKl7LqHzspnN05vCCcrQSXwz3746XVdo2PhrP7Ekr8pJR76ivHSsZJ0uriaeHssAbC2RU6MZztqHSEs20JZc1OsOXh7Kq2eLtFi39+VnfET60dLsfb7Y9/fDQwEdtplsvxvp5YiJdA6y6dEM4ua4v3YVVHfN8UL4AmFQvx7Wa0Em9598jp4WzK+j0wHv+j2NUS72+5Gl8XxYQF3NUS384L8agWlWL9bSusDLeZMtzYOxpfCEPxTVdD5XgvtlSeCme3l38SznqwPKMktSpe3m0mzTTgS9nHAAAAoAlRSxcAACCZ5+scPgAAABzK1VyHdBnwAQAANCB3N14GAABAPVeVQ7oAAAD55WKGDwAAIOdcVWeGDwAAINeY4QMAAMg1bssCAACQay6pmlDhY64d0wHfk2M9+h+PfP5YvuUk4nVwSi3x8kUT5Z5wtq20JpwdD7Z7fvvl4TZft6o9nN07Fi851Tsez16wLl6KbqAc30yftyyeXbV4fzy7dnc427G0P5ydGIyXbHv0wXiJuUVdg+FsKaGE4L4DS8LZpYt7w9kt2zeEcruGOsNt3rA9Xkbxzuoj4Wz3WHy/8NCur4ezC9s3hrMDo4+Gs5hdp3deGs6u81jJtGGPb7ttCX/G33NK/LO+cdmecPbAcPxzefE7vx3O2sJwVNX98cJh9sJ14Wzry6Z71TmkCwAAkGsuORdtAAAA5Fdtfo8ZPgAAgFxzzuEDAADIM67SBQAAyD1m+AAAAHKNq3QBAAByzcVVugAAADnnHNIFAADIOw7pAgAA5Jlz0cY0TBZ8y5aWeAmnajVehqalGC8Bk7YiPZysVMcS+hD73e6t3BJuc9veeFmzcR8OZ1P8+Ml4mbAJi6/fzp3x9Tti8VI8Hb4+nG1TKZwtKd6HcsI2liKlD4ta4r9bV0u83aFy7LM2Xo1/JvstXuKur/xUODtU2BfOFgrx2lCj5d5wtljoDmdTZiCq1YFwdj5oKcb/TpjF/9xVqqPh7D7tCGeHCn2h3HD1QLjNiWp8H73v8bPD2e7H4qUyB20knN344SvC2UWleCnUSsLusTNp5DNdGUMu2gAAAMg1LtoAAADIPZeY4QMAAMg3zuEDAADINc7hAwAAOA4w4AMAAMg3DukCAADkWXMd0o3fKAsAAAB1qg0+pmdml5jZw2a21cyumuT1NjP7x+z1O8xsw0xtMuADAABohHtjj2mYWVHSNZJeI+lMSZeb2ZmHxd4j6YC7nyrpM5I+OVNXGfABAAAk84b/m8F5kra6+2NeK7d1g6RLD8tcKukL2ddfkXSxmU1bmsR8hpHm0WRmeyU9cdjTyyX1HLNO4GhhvTUf1llzYr01H9ZZ85lqnZ3k7ism+wEz+072c41ol1Rfw2+Tu2/K2n2zpEvc/b3Z9++U9GJ3v7LuvR/IMjuy73+eZabc7o7pRRuTLTQzu9vdzz2W/cCRY701H9ZZc2K9NR/WWfNpZJ25+yWz1Z/ZwCFdAACA+WOnpPV136/Lnps0Y2Ytkrol7ZuuUQZ8AAAA88ddkjaa2clm1irpMkmbD8tslvSu7Os3S/q+z3CO3ny4D9+mue4AGsJ6az6ss+bEems+rLPmM2/WmbuXzexKSd+VVJR0rbtvMbOPSrrb3TdL+rykvzOzrZL2qzYonNYxvWgDAAAAxx6HdAEAAHKOAR8AAEDOzemAb6bSIZgfzOxaM9uT3ffn4HNLzexmM3s0+/+SuewjDmVm683sVjN70My2mNn7sudZb/OUmbWb2Z1m9pNsnf1R9vzJWemkrVkppda57isOZWZFM7vXzL6Vfc86m+fMbJuZ/dTM7jOzu7Pncr1/nLMBX7B0COaH6yQdfr+hqyTd4u4bJd2SfY/5oyzpA+5+pqTzJf1m9vlivc1fY5Je7u7Pl3S2pEvM7HzVSiZ9JiuhdEC1kkqYX94n6aG671lnzeFl7n523f33cr1/nMsZvkjpEMwD7n6balcB1asv6/IFSW84ln3C9Nx9l7v/OPt6QLU/RmvFepu3vGYw+7aUPVzSy1UrnSSxzuYdM1sn6Vcl/U32vYl11qxyvX+cywHfWknb677fkT2H5rDK3XdlX++WtGouO4OpmdkGSedIukOst3ktOzR4n6Q9km6W9HNJve5eziLsJ+ef/yPp9yVVs++XiXXWDFzSv5jZPWZ2RfZcrveP8+E+fGhy7u5mxv195iEz65L0VUnvd/f++trarLf5x90rks42s8WSvi7p2XPbI0zHzF4naY+732NmF81xd5DmQnffaWYrJd1sZj+rfzGP+8e5nOGLlA7B/PW0ma2RpOz/e+a4PziMmZVUG+z9g7t/LXua9dYE3L1X0q2SXiJpcVY6SWI/Od9cIOn1ZrZNtdOSXi7pz8U6m/fcfWf2/z2q/ePqPOV8/ziXA75I6RDMX/VlXd4l6Ztz2BccJjuP6POSHnL3T9e9xHqbp8xsRTazJzPrkPRK1c69vFW10kkS62xecfcPuvs6d9+g2t+w77v728U6m9fMrNPMFh78WtKrJD2gnO8f57TShpm9VrXzHw6WDvnjOesMpmRm10u6SNJySU9L+oikb0i6UdKJkp6Q9BZ3P/zCDswRM7tQ0r9K+ql+cW7Rh1Q7j4/1Ng+Z2VmqnSheVO0f4ze6+0fN7BTVZo+WSrpX0jvcfWzueorJZId0f9fdX8c6m9+y9fP17NsWSV9y9z82s2XK8f6R0moAAAA5R6UNAACAnGPABwAAkHMM+AAAAHKOAR8AAEDOMeADAADIOQZ8AAAAOceADwAAIOf+P7M9WyHk0rsAAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x720 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAnwAAAE1CAYAAAB9Uj1vAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAZHUlEQVR4nO3df7DldX3f8edrLywoID9coIQFwbi22aYGkx3E0RmJv7oSK2mbccCakA5284d0tDFtMe2oJc2MaUetmWFMNpWBmghFE+NOug1SxCHtVMoajLKL1JWA7GZlXUQFcYHd8+4f57v2sN57z/ece++ec773+WC+c78/PvdzPtzP7ve+9/MzVYUkSZK6a82kCyBJkqSVZcAnSZLUcQZ8kiRJHWfAJ0mS1HEGfJIkSR1nwCdJktRxBnySZkKSnUkunXQ5JGkWHTfpAkgSQJInBy6fDzwNHG6uf62q/u6xL5UkdUNceFnStEnyEPCOqvofky6LJHWBXbqSZkKSh5K8vjn/QJJPJfnDJE8k+WqSlyZ5b5L9SR5J8saB7z01yceT7EuyN8m/TzI3uf8bSTq2DPgkzap/AHwCOB24F7iN/jvtXOA64PcH0t4IHAJeArwceCPwjmNYVkmaKAM+SbPqL6rqtqo6BHwKOBP4YFU9C9wCXJDktCRnA5cB766qH1TVfuAjwBUTK7kkHWNO2pA0qx4dOP8hcKCqDg9cA5wM/ARwPLAvyZH0a4BHjkUhJWkaGPBJ6rpH6M/4Xde0BkrSqmOXrqROq6p9wOeADyV5QZI1SX4yyWsmXTZJOlYM+CStBr8CrAV2AY8DnwbOmWiJJOkYch0+SZKkjrOFT5IkqeMM+CRJkqZIkhuaReTvW+B5kvxukt1JvpLkZ4flacAnSZI0XW4ENi/y/E3AhubYAnxsWIYGfJIkSVOkqu4CvrNIksuB/1J9XwROS7LoRLRjug7fulPX1IvOaveRdeJJrfNdc8LprdP2Dh5onTa9Xuu0HHq2ddJae2L7Mjz+g3YJT2q/LWg90f7/K6eM8G+Cp9v/DDguw9P8qBDty1BPjTAJqdc+35zcPt/eE+3zffbg2tZpn3rmhNZpDx5uX4aDvfZ18fjhxd4/R3PJu2mwds0LWqc9Y679u+mEufbvkR8eav/n8fg17f+unXbiD4cnaswdf3h4osZxJx1snZYRXmWc0O7vex3f/r3Q2/9M67Rrjmv/M/jB909unXbt8e3f/XPHt38vHH6mfYhy6HD7tI8dbP8u/fah/Qeq6sz5nv39zS+rxw482TqvQV/60l/vBAb/oG2tqq0jZHEuz108fk9zb99C33BMA74XnXUcd3903p/bjzn4U5e0zvd5F/yj1mmffuCm1mnnnvp++7QHFvwZ/5hnLtjYOu2aW7/Y7vMvaR/0Pv359i/JE177vNZp2b23ddK8sP0vFk5on/bZe9u/TA49+fzWaZ/36va/AJ6845TWafd94/zWaf/ymxe2TvvA99q/rL/+RPvfWJ9+4ubWaQ8dfrx1Wq2cc57/ytZprzztpa3TvuSUp1qn/avH2/8Dfv3z2wcPb/7b97dOe/qZj7VOu+6VD7ROmxF+i9aL17dK9+xPvLh1ngc/+lDrtCee1f7v5Bdva79M5QXntn/3n3ZW+3p47G/Oap/2u+1/B970tfY/39/b/7sPL/iZB57k7h2/1TqvQcfl7QeratNY3zwmd9qQJEkaUVH0eu1bTZfZXuC8gev1zb0FLWkMX5LNSR5oZolcu5S8JEmSZkdRdWisYxlsA36lma17CfC9ZlehBY3dwpdkDrgeeAP9vuN7kmyrql3j5ilJkjQTCqpWpoUvyc3ApcC6JHuA9wPHA1TV7wHbgcuA3cBTwD8dludSunQvBnZX1YNN4W6hP2vEgE+SJHVaUfSWp7Xux/OuunLI8wLeOUqeSwn45psh8oqjEyXZQn+NGM4/s/1MUkmSpOlVy9U9e0ys+KSNZprxVoCf27DWjXslSVIHrJ6Ab+QZIpIkSZ1QRfVWR8B3D7AhyYX0A70rgLctS6kkSZKm3Wpo4auqQ0muAW4D5oAbqmrnspVMkiRpaq2eLl2qajv9qcGSJEmrSEFvhC1FJ8ydNiRJkkZUtYpa+CRJklanglUyaUOSJGl1KgM+SZKk7rNLV5IkqbtCEVv4JEmSOswuXUmSpK4z4JMkSeq4Io7hkyRJ6rACeocnXYrWDPgkSZJG5qQNSZKkjitb+CRJkjrNWbqSJEndF1v4JEmSOqzs0pUkSeo8W/gkSZI6zRY+SZKkTkuVLXySJEmdZ8AnSZLUYbbwSZIkrQIzFPCtmXQBJEmStLJs4ZMkSRpZkV5v0oVozYBPkiRpVMVMdeka8EmSJI3MdfgkSZI6L2WXriRJUne5l64kSdIq4KQNSZKkDqsy4JMkSeo6d9qQJEnqNFv4JEmSuq0w4JMkSeo2W/gkSZK6zZ02JEmSui3upStJkrQKGPBJkiR1mJM2JEmSus5JG5IkSd1WQK8mXYrW1ky6AJIkSTOp1xvvGCLJ5iQPJNmd5Np5np+f5M4k9yb5SpLLhuVpC58kSdLIVqZLN8kccD3wBmAPcE+SbVW1ayDZvwVuraqPJdkIbAcuWCxfAz5JkqRRrVyX7sXA7qp6ECDJLcDlwGDAV8ALmvNTgb8ZlqkBnyRJ0jhq7Ba+dUl2DFxvraqtzfm5wCMDz/YArzjq+z8AfC7JPwdOAl4/7AMN+CRJkkZWS2nhO1BVm5bw4VcCN1bVh5K8EvhEkp+uWjgCNeCTJEka1cp16e4Fzhu4Xt/cG3Q1sBmgqv53khOBdcD+hTJ1lq4kSdI4ejXesbh7gA1JLkyyFrgC2HZUmm8CrwNI8lPAicC3F8t0SS18SR4CngAOA4eW2DwpSZI0E6qWMoRvsXzrUJJrgNuAOeCGqtqZ5DpgR1VtA94D/EGSf0G/rfFXq2rRSHI5unR/vqoOLEM+kiRJs2OFFl6uqu30l1oZvPe+gfNdwKtGydMxfJIkSaMqYHZ2VlvyGL6iPy34S0m2zJcgyZYkO5LsOPC9GfrJSJIkLaY35jEBS23he3VV7U1yFnB7kq9V1V2DCZp1ZbYC/NyGtbOz6ZwkSdJiZiiqWVILX1Xtbb7uBz5Df3VoSZKkbiuoXsY6JmHsgC/JSUlOOXIOvBG4b7kKJkmSNNVWSZfu2cBnkhzJ55NV9efLUipJkqRpN6HWunGMHfA1m/r+zDKWRZIkaTY0XbqzwmVZJEmSRpbV0cInSZK0qpUBnyRJUnfZpStJkrQK9Ja6f8WxY8AnSZI0qnIMnyRJUufVDI3hm522SEmSJI3FFj5JkqRxOIZPkiSpu8pZupIkSV3npA1JkqTOm6VJGwZ8kiRJoyocwydJktR1juGTJEnqtNilK0mS1Gl26UqSJHWfXbqSJEkdVjhLV5IkqdsqdulKkiR1nV26kiRJHWeXriRJUpfZpStJktR9dulKkiR1mLN0JUmSuq5s4ZMkSeq4UOUYPkmSpG6zhU+SJKnbHMMnSZLUZY7hkyRJ6rZyDJ8kSVL32cInSZLUZeUYPkmSpM4z4JMkSeo4u3QlSZI6zEkbkiRJXeeyLJIkSd03S2P4ZqctUpIkaYpUZaxjmCSbkzyQZHeSaxdI89Yku5LsTPLJYXnawidJkjSqyop06SaZA64H3gDsAe5Jsq2qdg2k2QC8F3hVVT2e5Kxh+RrwSZIkjahYsS7di4HdVfUgQJJbgMuBXQNp/hlwfVU9DlBV+4dlapeuJEnSGKrWjHUA65LsGDi2DGR7LvDIwPWe5t6glwIvTfK/knwxyeZhZbWFT5IkaQy98Vv4DlTVpiV89HHABuBSYD1wV5K/V1XfXewbJEmSNIoVGsMH7AXOG7he39wbtAe4u6qeBf46yf+lHwDes1CmdulKkiSN6MgYvhWYpXsPsCHJhUnWAlcA245K86f0W/dIso5+F++Di2VqC58kSdIYVmLSRlUdSnINcBswB9xQVTuTXAfsqKptzbM3JtkFHAb+ZVU9tli+BnySJEljWKmFl6tqO7D9qHvvGzgv4NeboxUDPkmSpFFV6M3QXrpDS5rkhiT7k9w3cO+MJLcn+Xrz9fSVLaYkSZLG1SY0vRE4en2Xa4E7qmoDcEdzLUmStCoUUL2MdUzC0ICvqu4CvnPU7cuBm5rzm4BfXN5iSZIkTbeV2kt3JYw7hu/sqtrXnH8LOHuhhM3q0VsAzj9zbsyPkyRJmi6TCt7GseTRhs1MkVrk+daq2lRVm9adOjuDGyVJkhZU/Z02xjkmYdwWvkeTnFNV+5KcAwzdtFeSJKkrisl1z45j3Ca3bcBVzflVwGeXpziSJEmzoVNj+JLcTH/7jnVJ9gDvBz4I3JrkauBh4K0rWUhJkqRpM6nu2XEMDfiq6soFHr1umcsiSZI0M2apS9edNiRJkkZUZcAnSZLUcZObcTsOAz5JkqQx2MInSZLUcQZ8kiRJHVZ0bJauJEmSjuKkDUmSpK5z0oYkSVKnFf3t1WaFAZ8kSdIY7NKVJEnqOLt0JUmSOi228EmSJHVZlS18kiRJnWcLnyRJUsf1nKUrSZLUXYUtfJIkSR3nwsuSJEmdZwufJElShxXQm3QhRmDAJ0mSNKqyhU+SJKnzHMMnSZLUceWyLJIkSd1VztKVJEnqvl5NugTtGfBJkiSNwS5dSZKkDiuctCFJktRtBWWXriRJUrf17NKVJEnqrsKFlyVJkjrOZVkkSZI6b4aG8BnwSZIkjWrWZumumXQBJEmStLJs4ZMkSRpDb9IFGIEBnyRJ0hhmaZauXbqSJEkjquqP4RvnGCbJ5iQPJNmd5NpF0v3jJJVk07A8DfgkSZLGUGMei0kyB1wPvAnYCFyZZOM86U4B3gXc3aasBnySJEljWKEWvouB3VX1YFU9A9wCXD5Put8Cfgc42KasBnySJEkjKvqTNsY5gHVJdgwcWwayPhd4ZOB6T3PvR5L8LHBeVf23tuV10oYkSdLIspRJGweqaui4u3k/NVkDfBj41VG+z4BPkiRpDCu0LMte4LyB6/XNvSNOAX4a+EISgL8FbEvylqrasVCmBnySJEkjKlZsWZZ7gA1JLqQf6F0BvO1Hn1v1PWDdkeskXwB+Y7FgDwz4JEmSxtJbgc10q+pQkmuA24A54Iaq2pnkOmBHVW0bJ18DPkmSpDGsQLzXz7dqO7D9qHvvWyDtpW3yNOCTJEka0ZGFl2eFAZ8kSdIY3EtXkiSp4zq1l26SG5LsT3LfwL0PJNmb5MvNcdnKFlOSJGl6LHHh5WOuzU4bNwKb57n/kaq6qDm2z/NckiSps6rGOyZhaJduVd2V5IJjUBZJkqSZ0aNDXbqLuCbJV5ou39MXSpRky5G94g58b5aGN0qSJM2v6K/DN84xCeMGfB8DfhK4CNgHfGihhFW1tao2VdWmdacuJb6UJEmaHp3q0p1PVT165DzJHwB/tmwlkiRJmnrpfpduknMGLv8hcN9CaSVJkjpnzNa9qW3hS3IzcCmwLske4P3ApUkuot+F/RDwaytXREmSpOlyZFmWWdFmlu6V89z++AqURZIkaWZMagLGONxpQ5IkaQwzFO8Z8EmSJI2qvyzL7EzaMOCTJEkaw6QmYIzDgE+SJGkMnZq0IUmSpOcqbOGTJEnqPFv4JEmSumyC++KOw4BPkiRpRIXLskiSJHWeLXySJEkd56QNSZKkDuvcXrqSJEn6cXbpSpIkddwMxXsGfJIkSaPq76U76VK0Z8AnSZI0qnLShiRJUufN0qSNNZMugCRJklaWLXySJEkjcgyfJEnSKjBD8Z4BnyRJ0jhs4ZMkSeo4Z+lKkiR1mFurSZIkrQK9GWriM+CTJEkaw+yEewZ8kiRJI6ty0oYkSVLHFTVDbXwGfJIkSSNy4WVJkqRVwFm6kiRJHVfO0pUkSeou1+GTJElaBWaphW/NpAsgSZI0i3pjHsMk2ZzkgSS7k1w7z/NfT7IryVeS3JHkRcPyNOCTJEkaUX+Wbo11LCbJHHA98CZgI3Blko1HJbsX2FRVLwM+DfyHYeU14JMkSRpDjfnfEBcDu6vqwap6BrgFuPw5n1t1Z1U91Vx+EVg/LFPH8EmSJI1hCZM21iXZMXC9taq2NufnAo8MPNsDvGKRvK4G/vuwDzTgkyRJGlFR9MbfaeNAVW1aahmSvB3YBLxmWFoDPkmSpFEVQ8fjjWkvcN7A9frm3nMkeT3wb4DXVNXTwzI14JMkSRrDCu2lew+wIcmF9AO9K4C3DSZI8nLg94HNVbW/TaYGfJIkSSPqL7y8/AFfVR1Kcg1wGzAH3FBVO5NcB+yoqm3AfwROBj6VBOCbVfWWxfI14JMkSRrDSgR8AFW1Hdh+1L33DZy/ftQ8DfgkSZJG1mqJlalhwCdJkjSilerSXSkGfJIkSaMK9LKElfiOMQM+SZKkMdjCJ0mS1GHVLL08K4bupZvkvCR3JtmVZGeSdzX3z0hye5KvN19PX/niSpIkTYdes9vGqMckDA34gEPAe6pqI3AJ8M4kG4FrgTuqagNwR3MtSZK0KvTSG+uYhKFdulW1D9jXnD+R5H76G/teDlzaJLsJ+ALwr1eklJIkSVOk36E7O126I43hS3IB8HLgbuDsJhgE+BZw9vIWTZIkaXp1MuBLcjLwx8C7q+r7zVYeAFRVJZm3UzrJFmALwPlnzi2ttJIkSVOhY5M2AJIcTz/Y+6Oq+pPm9qNJzmmenwPMu3lvVW2tqk1VtWndqa0+TpIkaaoVszWGr80s3QAfB+6vqg8PPNoGXNWcXwV8dvmLJ0mSNI36Y/jG+W8S2nTpvgr4ZeCrSb7c3PtN4IPArUmuBh4G3roiJZQkSZpCxeFJF6G1NrN0/yeQBR6/bnmLI0mSNP06PUtXkiRJfbMU8DmLQpIkqeNs4ZMkSRpZdWsMnyRJkp6rmK0uXQM+SZKkMczSwssGfJIkSSMrenbpSpIkdVdhC58kSVLHFb2yhU+SJKnTbOGTJEnqNJdlkSRJ6rQCemULnyRJUoeVXbqSJEmdVlBO2pAkSequfvueLXySJEmdVo7hkyRJ6jJn6UqSJHWeLXySJEmd5ixdSZKkTiucpStJktRxZZeuJElS19mlK0mS1GXlpA1JkqSOc9KGJElSpzlpQ5IkqfMKbOGTJEnqNsfwSZIkdZpj+CRJklYBAz5JkqRus0tXkiSpy2arS3fNpAsgSZI0m3pjHotLsjnJA0l2J7l2nucnJPmvzfO7k1wwLE8DPkmSpHFUjXcsIskccD3wJmAjcGWSjUcluxp4vKpeAnwE+J1hRTXgkyRJGlmN/d8QFwO7q+rBqnoGuAW4/Kg0lwM3NeefBl6XJItlmhoSaS6nJN8GHj7q9jrgwDErhJaL9TZ7rLPZZL3NHuts9ixUZy+qqjPn+4Ykf9583zhOBA4OXG+tqq1Nvr8EbK6qdzTXvwy8oqquGfjs+5o0e5rrbzRpFvxzd0wnbcz3Q0uyo6o2HctyaOmst9ljnc0m6232WGezZ5w6q6rNK1WelWCXriRJ0vTYC5w3cL2+uTdvmiTHAacCjy2WqQGfJEnS9LgH2JDkwiRrgSuAbUel2QZc1Zz/EvD5GjJGbxrW4ds66QJoLNbb7LHOZpP1Nnuss9kzNXVWVYeSXAPcBswBN1TVziTXATuqahvwceATSXYD36EfFC7qmE7akCRJ0rFnl64kSVLHGfBJkiR13EQDvmFbh2g6JLkhyf5m3Z8j985IcnuSrzdfT59kGfVcSc5LcmeSXUl2JnlXc996m1JJTkzyf5L8VVNn/665f2GzddLuZiultZMuq54ryVySe5P8WXNtnU25JA8l+WqSLyfZ0dzr9PtxYgFfy61DNB1uBI5eb+ha4I6q2gDc0VxrehwC3lNVG4FLgHc2f7+st+n1NPDaqvoZ4CJgc5JL6G+Z9JFmC6XH6W+ppOnyLuD+gWvrbDb8fFVdNLD+Xqffj5Ns4WuzdYimQFXdRX8W0KDBbV1uAn7xWJZJi6uqfVX1l835E/R/GZ2L9Ta1qu/J5vL45ijgtfS3TgLrbOokWQ/8AvCfm+tgnc2qTr8fJxnwnQs8MnC9p7mn2XB2Ve1rzr8FnD3JwmhhSS4AXg7cjfU21ZquwS8D+4HbgW8A362qQ00S35PT5z8B/wroNdcvxDqbBQV8LsmXkmxp7nX6/TgN6/BpxlVVJXF9nymU5GTgj4F3V9X3B/fWtt6mT1UdBi5KchrwGeDvTLZEWkySNwP7q+pLSS6dcHE0mldX1d4kZwG3J/na4MMuvh8n2cLXZusQTa9Hk5wD0HzdP+Hy6ChJjqcf7P1RVf1Jc9t6mwFV9V3gTuCVwGnN1knge3LavAp4S5KH6A9Lei3wUayzqVdVe5uv++n/4+piOv5+nGTA12brEE2vwW1drgI+O8Gy6CjNOKKPA/dX1YcHHllvUyrJmU3LHkmeB7yB/tjLO+lvnQTW2VSpqvdW1fqquoD+77DPV9U/wTqbaklOSnLKkXPgjcB9dPz9ONGdNpJcRn/8w5GtQ357YoXRgpLcDFwKrAMeBd4P/ClwK3A+8DDw1qo6emKHJiTJq4G/AL7K/x9b9Jv0x/FZb1MoycvoDxSfo/+P8Vur6rokL6bfenQGcC/w9qp6enIl1XyaLt3fqKo3W2fTramfzzSXxwGfrKrfTvJCOvx+dGs1SZKkjnOnDUmSpI4z4JMkSeo4Az5JkqSOM+CTJEnqOAM+SZKkjjPgkyRJ6jgDPkmSpI77f20z8hkklM02AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x720 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "model.show_gradcam(xb[0], yb[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([16, 5, 2])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "create_conv_lin_3d_head(\n",
       "  (0): BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (1): Conv1d(128, 5, kernel_size=(1,), stride=(1,), bias=False)\n",
       "  (2): Transpose(-1, -2)\n",
       "  (3): BatchNorm1d(12, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (4): Transpose(-1, -2)\n",
       "  (5): Linear(in_features=12, out_features=2, bias=False)\n",
       ")"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bs = 16\n",
    "n_vars = 3\n",
    "seq_len = 12\n",
    "c_out = 10\n",
    "xb = torch.rand(bs, n_vars, seq_len)\n",
    "new_head = partial(conv_lin_3d_head, d=(5, 2))\n",
    "net = XCM(n_vars, c_out, seq_len, custom_head=new_head)\n",
    "print(net.to(xb.device)(xb).shape)\n",
    "net.head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([16, 2])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Sequential(\n",
       "  (0): AdaptiveAvgPool1d(output_size=1)\n",
       "  (1): Flatten(full=False)\n",
       "  (2): BatchNorm1d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (3): Linear(in_features=128, out_features=512, bias=False)\n",
       "  (4): ReLU(inplace=True)\n",
       "  (5): BatchNorm1d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (6): Linear(in_features=512, out_features=2, bias=False)\n",
       ")"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bs = 16\n",
    "n_vars = 3\n",
    "seq_len = 12\n",
    "c_out = 2\n",
    "xb = torch.rand(bs, n_vars, seq_len)\n",
    "net = XCM(n_vars, c_out, seq_len)\n",
    "change_model_head(net, create_pool_plus_head, concat_pool=False)\n",
    "print(net.to(xb.device)(xb).shape)\n",
    "net.head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "from tsai.imports import create_scripts\n",
    "from tsai.export import get_nb_name\n",
    "nb_name = get_nb_name()\n",
    "create_scripts(nb_name);"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
